{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys, os, _pickle as pickle\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import nltk\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "data_dir = 'data'\n",
    "ckpt_dir = 'checkpoint'\n",
    "word_embd_dir = 'checkpoint/word_embd'\n",
    "model_dir = 'checkpoint/model3v1'\n",
    "\n",
    "word_embd_dim = 100\n",
    "pos_embd_dim = 25\n",
    "dep_embd_dim = 25\n",
    "\n",
    "word_vocab_size = 400001\n",
    "pos_vocab_size = 10\n",
    "dep_vocab_size = 21\n",
    "\n",
    "relation_classes = 10\n",
    "\n",
    "state_size = 100\n",
    "batch_size = 10\n",
    "channels = 3\n",
    "\n",
    "max_len_seq = 90\n",
    "max_len_path = 20\n",
    "max_num_childs = 20\n",
    "\n",
    "lambda_l2 = 0.0001\n",
    "init_learning_rate = 0.001\n",
    "decay_steps = 2000\n",
    "decay_rate = 0.96\n",
    "gradient_clipping = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"input\"):\n",
    "    \n",
    "    fp_length = tf.placeholder(tf.int32, shape=[batch_size], name=\"fp_ength\")\n",
    "    fp = tf.placeholder(tf.int32, [batch_size, 2, max_len_seq], name=\"full_path\")\n",
    "    \n",
    "    sp_length = tf.placeholder(tf.int32, shape=[batch_size, 2], name=\"sp_length\")\n",
    "    sp = tf.placeholder(tf.int32, [batch_size, 2, max_len_path], name=\"shortest_path\")\n",
    "    \n",
    "    sp_pos = tf.placeholder(tf.int32, [batch_size, 2, max_len_path], name=\"sp_pos\")\n",
    "    \n",
    "    sp_childs = tf.placeholder(tf.int32, [batch_size, 2, max_len_path, max_num_childs], name=\"sp_childs\")\n",
    "    sp_num_childs = tf.placeholder(tf.int32, [batch_size, 2, max_len_path], name=\"sp_num_childs\")\n",
    "    \n",
    "    relation = tf.placeholder(tf.int32, [batch_size], name=\"relation\")\n",
    "    y_entity = tf.placeholder(tf.int32, [batch_size, max_len_seq], name=\"y_enity\")\n",
    "\n",
    "with tf.name_scope(\"word_embedding\"):\n",
    "    W = tf.Variable(tf.constant(0.0, shape=[word_vocab_size, word_embd_dim]), name=\"W\")\n",
    "    embedding_placeholder = tf.placeholder(tf.float32,[word_vocab_size, word_embd_dim])\n",
    "    embedding_init = W.assign(embedding_placeholder)\n",
    "    embd_fp_word = tf.nn.embedding_lookup(W,fp[:,0])\n",
    "    word_embedding_saver = tf.train.Saver({\"word_embedding/W\": W})\n",
    "\n",
    "with tf.name_scope(\"pos_embedding\"):\n",
    "    W = tf.Variable(tf.random_uniform([pos_vocab_size, pos_embd_dim]), name=\"W\")\n",
    "    embd_fp_pos = tf.nn.embedding_lookup(W, fp[:,1])\n",
    "    pos_embedding_saver = tf.train.Saver({\"pos_embedding/W\": W})\n",
    "\n",
    "with tf.name_scope(\"dep_embedding\"):\n",
    "    W = tf.Variable(tf.random_uniform([dep_vocab_size, dep_embd_dim]), name=\"W\")\n",
    "    embd_sp = tf.nn.embedding_lookup(W, sp)\n",
    "    dep_embedding_saver = tf.train.Saver({\"dep_embedding/W\": W})\n",
    "    \n",
    "embd_fp = tf.concat([embd_fp_word, embd_fp_pos], axis=2)\n",
    "embd_fp_rev = tf.reverse(embd_fp, [1])\n",
    "fp_length_rev = tf.reverse(fp_length, [0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lstm_seq_init(channel, embedding_dim, state_size):\n",
    "    init_const = tf.zeros([1, state_size])\n",
    "    with tf.variable_scope(channel):\n",
    "        W_i = tf.get_variable(\"W_i\",shape=[embedding_dim, state_size] ,initializer=tf.contrib.layers.xavier_initializer())\n",
    "        U_i = tf.get_variable(\"U_i\",shape=[state_size, state_size] ,initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_i = tf.get_variable(\"b_i\", initializer=init_const)\n",
    "\n",
    "        W_f = tf.get_variable(\"W_f\",shape=[embedding_dim, state_size] ,initializer=tf.contrib.layers.xavier_initializer())\n",
    "        U_f = tf.get_variable(\"U_f\",shape=[state_size, state_size] ,initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_f = tf.get_variable(\"b_f\", initializer=init_const)\n",
    "\n",
    "        W_o = tf.get_variable(\"W_o\",shape=[embedding_dim, state_size] ,initializer=tf.contrib.layers.xavier_initializer())\n",
    "        U_o = tf.get_variable(\"U_o\",shape=[state_size, state_size] ,initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_o = tf.get_variable(\"b_o\", initializer=init_const)\n",
    "\n",
    "        W_g = tf.get_variable(\"W_g\",shape=[embedding_dim, state_size] ,initializer=tf.contrib.layers.xavier_initializer())\n",
    "        U_g = tf.get_variable(\"U_g\",shape=[state_size, state_size] ,initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_g = tf.get_variable(\"b_g\", initializer=init_const)\n",
    "\n",
    "lstm_seq_init(\"lstm_fw\", word_embd_dim + pos_embd_dim, state_size)\n",
    "lstm_seq_init(\"lstm_bw\", word_embd_dim + pos_embd_dim, state_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cond(i, steps, *agrs):\n",
    "    return i< steps\n",
    "\n",
    "init_state_seq = tf.zeros([2, 1, state_size])\n",
    "\n",
    "x = tf.constant(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lstm_seq(input_embd, seq_len, scope):\n",
    "    def body(j, const, steps, input_embd, states_seq, states_series):\n",
    "        inputs = tf.expand_dims(input_embd[j], [0])\n",
    "        \n",
    "        hs = states_seq[0]\n",
    "        cs = states_seq[1]\n",
    "        \n",
    "        hs_ = states_series[0]\n",
    "        cs_ = states_series[1]\n",
    "        with tf.variable_scope(scope, reuse=True):\n",
    "            W_i = tf.get_variable(\"W_i\")\n",
    "            U_i = tf.get_variable(\"U_i\")\n",
    "            b_i = tf.get_variable(\"b_i\")\n",
    "\n",
    "            W_f = tf.get_variable(\"W_f\")\n",
    "            U_f = tf.get_variable(\"U_f\")\n",
    "            b_f = tf.get_variable(\"b_f\")\n",
    "\n",
    "            W_o = tf.get_variable(\"W_o\")\n",
    "            U_o = tf.get_variable(\"U_o\")\n",
    "            b_o = tf.get_variable(\"b_o\")\n",
    "\n",
    "            W_g = tf.get_variable(\"W_g\")\n",
    "            U_g = tf.get_variable(\"U_g\")\n",
    "            b_g = tf.get_variable(\"b_g\")\n",
    "            \n",
    "            input_gate = tf.sigmoid(tf.matmul(inputs, W_i) + tf.matmul(hs, U_i) + b_i)\n",
    "            forget_gate = tf.sigmoid(tf.matmul(inputs, W_f) + tf.matmul(hs, U_f) + b_f)\n",
    "            output_gate = tf.sigmoid(tf.matmul(inputs, W_o) + tf.matmul(hs, U_o) + b_o)\n",
    "            gt = tf.tanh(tf.matmul(inputs, W_g) + tf.matmul(hs, U_g) + b_g)\n",
    "            cs = input_gate * gt + forget_gate * cs\n",
    "            hs = output_gate * tf.tanh(cs)\n",
    "            \n",
    "            hs_ = tf.cond(tf.equal(j, const), lambda: hs, lambda: tf.concat([hs_, hs], 0))\n",
    "            cs_ = tf.cond(tf.equal(j, const), lambda: cs, lambda: tf.concat([cs_, cs], 0))\n",
    "            \n",
    "            states_seq = tf.stack([hs, cs], axis=0)\n",
    "            states_series = tf.stack([hs_, cs_], axis=0)\n",
    "            \n",
    "            return j+1, const, steps, input_embd, states_seq, states_series\n",
    "\n",
    "    _, _, _, _, _, state_series_seq = tf.while_loop(cond, body, \n",
    "            [0, 0, seq_len, input_embd, init_state_seq, init_state_seq],\n",
    "            shape_invariants=[x.get_shape(), x.get_shape(), x.get_shape(), input_embd.get_shape(),\n",
    "            init_state_seq.get_shape(), \n",
    "            tf.TensorShape([2, None, state_size])])\n",
    "    \n",
    "    return state_series_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'concat_10:0' shape=(10, ?, 200) dtype=float32>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states_series_ = []\n",
    "states_series_rev = []\n",
    "\n",
    "for b in range(batch_size):\n",
    "    seq_len = fp_length[b]\n",
    "    input_embd = embd_fp[b]   \n",
    "    states_series_.append(lstm_seq(input_embd, seq_len, \"lstm_fw\"))\n",
    "    \n",
    "    input_embd = embd_fp_rev[b]   \n",
    "    seq_len = fp_length_rev[b]\n",
    "    states_series_rev.append(lstm_seq(input_embd, seq_len, \"lstm_bw\"))\n",
    "    \n",
    "states_series_bw = tf.reverse(states_series_rev, [2])\n",
    "\n",
    "for i in range(batch_size):\n",
    "    if(i==0):\n",
    "        states_series_fw = tf.expand_dims(states_series_[i], [0])\n",
    "    else:\n",
    "        states_series_fw = tf.concat([states_series_fw, tf.expand_dims(states_series_[i], [0])], axis=0)\n",
    "        \n",
    "hidden_states_seq = tf.concat([states_series_fw[:,0], states_series_bw[0:,1]], 2)\n",
    "\n",
    "hidden_states_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def lstm_dep_init(channel, dep_input_size, state_size):\n",
    "    init_const = tf.zeros([1, state_size])\n",
    "\n",
    "    with tf.variable_scope(channel):\n",
    "        W_i = tf.get_variable(\"W_i\", shape=[dep_input_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        U_i = tf.get_variable(\"U_i\", shape=[state_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_i = tf.get_variable(\"b_i\", initializer=init_const)\n",
    "        U_it = tf.get_variable(\"U_it\", shape=[state_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "\n",
    "        W_f = tf.get_variable(\"W_f\", shape=[dep_input_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        U_f = tf.get_variable(\"U_f\", shape=[state_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_f = tf.get_variable(\"b_f\", initializer=init_const)\n",
    "        U_fsp = tf.get_variable(\"U_fsp\", shape=[2, state_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        U_ffp = tf.get_variable(\"U_ffp\", shape=[max_num_childs, state_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "\n",
    "        W_o = tf.get_variable(\"W_o\", shape=[dep_input_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        U_o = tf.get_variable(\"U_o\", shape=[state_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_o = tf.get_variable(\"b_o\", initializer=init_const)\n",
    "        U_ot = tf.get_variable(\"U_ot\", shape=[state_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "\n",
    "        W_u = tf.get_variable(\"W_u\", shape=[dep_input_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        U_u = tf.get_variable(\"U_u\", shape=[state_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "        b_u = tf.get_variable(\"b_u\", initializer=init_const)\n",
    "        U_ut = tf.get_variable(\"U_ut\", shape=[state_size, state_size], initializer=tf.contrib.layers.xavier_initializer())\n",
    "\n",
    "dep_input_size = state_size * 2 + dep_embd_dim\n",
    "\n",
    "lstm_dep_init(\"lstm_btup\", dep_input_size, state_size)\n",
    "lstm_dep_init(\"lstm_tpdn\", dep_input_size, state_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init_state = tf.zeros([2, 1, 1, state_size])\n",
    "\n",
    "def lstm_dep(b, p, start, seq_len, input_embd, input_pos, input_childs, input_num_child, states_seq, init_state_dep, scope):  \n",
    "    \n",
    "    def loop_over_seq(index, const, steps, input_pos, input_embd, input_childs, input_num_child, states_seq, states_dep, states_series):\n",
    "        \n",
    "        inputs = tf.expand_dims(tf.concat([hidden_states_seq[b][input_pos[p][index]], input_embd[p][index]],0),0)\n",
    "        childs = input_childs[p][index]\n",
    "        num_child = input_num_child[p][index]    \n",
    "        num_child_sp = tf.shape(states_dep[0])[0]\n",
    "        \n",
    "        with tf.variable_scope(scope, reuse=True):\n",
    "            W_i = tf.get_variable(\"W_i\")\n",
    "            U_i = tf.get_variable(\"U_i\")\n",
    "            b_i = tf.get_variable(\"b_i\")\n",
    "            U_it = tf.get_variable(\"U_it\")\n",
    "\n",
    "            W_f = tf.get_variable(\"W_f\")\n",
    "            U_f = tf.get_variable(\"U_f\")\n",
    "            b_f = tf.get_variable(\"b_f\")\n",
    "            U_fsp = tf.get_variable(\"U_fsp\")\n",
    "            U_ffp = tf.get_variable(\"U_ffp\")\n",
    "            \n",
    "            W_o = tf.get_variable(\"W_o\")\n",
    "            U_o= tf.get_variable(\"U_o\")\n",
    "            b_o = tf.get_variable(\"b_o\")\n",
    "            U_ot = tf.get_variable(\"U_ot\")\n",
    "\n",
    "            W_u = tf.get_variable(\"W_u\")\n",
    "            U_u = tf.get_variable(\"U_u\")\n",
    "            b_u = tf.get_variable(\"b_u\")\n",
    "            U_ut = tf.get_variable(\"U_ut\")    \n",
    "\n",
    "            it = tf.matmul(inputs, W_i) + b_i + tf.matmul(states_dep[0][0], U_i)\n",
    "            ft = tf.matmul(inputs, W_f) + b_f + tf.matmul(states_dep[0][0], U_f)\n",
    "            ot = tf.matmul(inputs, W_o) + b_o + tf.matmul(states_dep[0][0], U_o)\n",
    "            ut = tf.matmul(inputs, W_u) + b_u + tf.matmul(states_dep[0][0], U_u)\n",
    "\n",
    "            def matmul(k, steps, it, ft, ot, ut):\n",
    "                it += tf.matmul(states_dep[0][k], U_i)\n",
    "                ft += tf.matmul(states_dep[0][k], U_f) \n",
    "                ot += tf.matmul(states_dep[0][k], U_o) \n",
    "                ut += tf.matmul(states_dep[0][k], U_u) \n",
    "                return k+1, steps, it, ft, ot, ut\n",
    "            \n",
    "            _, _, it, ft, ot, ut = tf.while_loop(cond, matmul, [1, num_child_sp, it, ft, ot, ut])\n",
    "\n",
    "\n",
    "            def child_sum(k, steps, out, U):\n",
    "                out += tf.matmul(states_seq[0][childs[k]], U)\n",
    "                return k+1, steps, out, U\n",
    "\n",
    "            _, _, ht_i, _ = tf.while_loop(cond, child_sum, [0, num_child, it, U_it])\n",
    "            _, _, ht_o, _ = tf.while_loop(cond, child_sum, [0, num_child, ot, U_ot])\n",
    "            _, _, ht_u, _ = tf.while_loop(cond, child_sum, [0, num_child, ut, U_ut])\n",
    "\n",
    "            input_gate = tf.sigmoid(ht_i)\n",
    "            output_gate = tf.sigmoid(ht_o)\n",
    "            u_input = tf.tanh(ht_u)\n",
    "\n",
    "            cell_state = input_gate * u_input \n",
    "\n",
    "            def cell_state_sp(k, steps, cell_state):\n",
    "                _, _, f_sp, _ = tf.while_loop(cond, child_sum, [0, num_child, ft, U_fsp[k]])\n",
    "                cell_state += tf.sigmoid(f_sp) * states_dep[1][k]\n",
    "                return k+1, steps, cell_state\n",
    "            \n",
    "            _, _, cell_state = tf.while_loop(cond, cell_state_sp, [0, num_child_sp, cell_state])\n",
    "\n",
    "            def cell_states_fp(k, steps, ctl):\n",
    "                _, _, f_fp, _ = tf.while_loop(cond, child_sum, [i, num_child, ft, U_ffp[k]])\n",
    "                ctl += tf.sigmoid(f_fp) * states_seq[1][childs[k]]\n",
    "                return k+1, steps, ctl\n",
    "\n",
    "            _, _, cds = tf.while_loop(cond, cell_states_fp, [0, num_child, cell_state])\n",
    "            \n",
    "            hds = tf.expand_dims(output_gate * tf.tanh(cds), 0)\n",
    "\n",
    "            cds = tf.expand_dims(cds, 0)\n",
    "            \n",
    "            states_dep = tf.stack([hds, cds], axis=0)\n",
    "\n",
    "            hds_ = tf.cond(tf.equal(i, const), lambda: states_dep[0],\n",
    "                           lambda: tf.concat([states_series[0], states_dep[0]], 0))\n",
    "            \n",
    "            cds_ = tf.cond(tf.equal(i, const), lambda: states_dep[1],\n",
    "                           lambda: tf.concat([states_series[1], states_dep[1]], 0))\n",
    "\n",
    "            states_series = tf.stack([hds_, cds_], axis=0)\n",
    "\n",
    "        return index+1, const, steps, input_pos, input_embd, input_childs, input_num_child, states_seq, states_dep, states_series\n",
    "    \n",
    "    _, _, _, _, _, _, _, _, _, states_series_dep = tf.while_loop(cond, loop_over_seq,[start, start, seq_len,\n",
    "                            input_pos, input_embd, input_childs, input_num_child, states_seq, init_state_dep,\n",
    "                            init_state], shape_invariants=[x.get_shape(), x.get_shape(), x.get_shape(),\n",
    "                            input_pos.get_shape(), input_embd.get_shape(), input_childs.get_shape(),\n",
    "                            input_num_child.get_shape(), states_seq.get_shape(),\n",
    "                            tf.TensorShape([2, None, 1, state_size]), tf.TensorShape([2, None, 1, state_size])])\n",
    "    \n",
    "    return states_series_dep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'concat_39:0' shape=(10, 300) dtype=float32>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lca_series_btup = []\n",
    "dp_series_tpdn = []\n",
    "\n",
    "for i in range(batch_size):\n",
    "    \n",
    "    input_pos = sp_pos[i]\n",
    "    input_embd = embd_sp[i]\n",
    "    input_childs = sp_childs[i]\n",
    "    input_num_child = sp_num_childs[i]\n",
    "    \n",
    "    input_pos_rev = tf.reverse(sp_pos[i], [1])\n",
    "    input_embd_rev = tf.reverse(embd_sp[i], [1])\n",
    "    input_childs_rev = tf.reverse(sp_childs[i], [1])\n",
    "    input_num_child_rev = tf.reverse(sp_num_childs[i], [1])\n",
    "    \n",
    "    states_seq_fw = tf.expand_dims(states_series_fw[i], 2)\n",
    "    states_seq_bw = tf.expand_dims(states_series_bw[i], 2)\n",
    "    \n",
    "    s1 = lstm_dep(i, 0, 0, sp_length[i][0]-1, input_embd, input_pos, input_childs, input_num_child,\n",
    "                 states_seq_fw, init_state, \"lstm_btup\")\n",
    "    \n",
    "    lca_btup = tf.cond(sp_length[i][0]>1, lambda: s1[:,sp_length[i][0]-2], lambda: init_state[:,0])\n",
    "\n",
    "    s2 = lstm_dep(i, 1, 0, sp_length[i][1], input_embd, input_pos, input_childs, input_num_child,\n",
    "                 states_seq_bw, init_state, \"lstm_btup\")\n",
    "    \n",
    "\n",
    "    lca_btup = tf.cond(sp_length[i][1]>0, lambda: tf.stack([lca_btup,\n",
    "                      s2[:,sp_length[i][0]+sp_length[i][1]-2]],axis=1), lambda: tf.expand_dims(lca_btup, 1))\n",
    "\n",
    "    lca_series_btup.append(lstm_dep(i, 0, sp_length[i][0]-1, sp_length[i][0], input_embd, input_pos, input_childs,\n",
    "                                   input_num_child, states_seq_fw, lca_btup, \"lstm_btup\")[0,0])\n",
    "    \n",
    "    lca_tpdn = lstm_dep(i, 0, 0, 1, input_embd_rev, input_pos_rev, input_childs_rev, input_num_child_rev, \n",
    "                        states_seq_bw, init_state, \"lstm_tpdn\")\n",
    "    \n",
    "    dp1 = lstm_dep(i, 0, 1, sp_length[i][0], input_embd_rev, input_pos_rev, input_childs_rev, input_num_child_rev,\n",
    "                   states_seq_bw, lca_tpdn, \"lstm_tpdn\")[0,-1]\n",
    "    \n",
    "    dp2 = lstm_dep(i, 1, 0, sp_length[i][1], input_embd_rev, input_pos_rev, input_childs_rev, input_num_child_rev,\n",
    "                   states_seq_bw, lca_tpdn, \"lstm_tpdn\")[0,-1]\n",
    "    \n",
    "    \n",
    "    dp_series_tpdn.append(tf.concat([dp1, dp2], 1))\n",
    "    \n",
    "    \n",
    "\n",
    "for i in range(batch_size):\n",
    "    \n",
    "    temp = tf.concat([lca_series_btup[i], dp_series_tpdn[i]], 1)\n",
    "    if(i==0):\n",
    "        dp_series = temp\n",
    "    else:\n",
    "        dp_series = tf.concat([dp_series,temp], axis=0)\n",
    "\n",
    "dp_series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"hidden_layer_seq\"):\n",
    "    W = tf.Variable(tf.truncated_normal([200, 100], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([100]), name=\"b\")\n",
    "    \n",
    "    y_hidden_layer = []\n",
    "    y_hl = tf.zeros([1, 100])\n",
    "    \n",
    "    for batch in range(batch_size):\n",
    "        s_seq = tf.expand_dims(hidden_states_seq[batch], 1)\n",
    "        \n",
    "        def matmul_hl(j, const, steps, input_seq, out_seq):\n",
    "            temp = tf.tanh(tf.matmul(input_seq[j], W) + b)\n",
    "            out_seq = tf.cond(tf.equal(j, const), lambda: temp, lambda: tf.concat([out_seq, temp], 0))\n",
    "            return j+1, const, steps, input_seq, out_seq\n",
    "        \n",
    "        _, _, _, _, output_seq = tf.while_loop(cond, matmul_hl, \n",
    "                                [0, 0, fp_length[batch], s_seq, y_hl],\n",
    "                                shape_invariants=[x.get_shape(), x.get_shape(), x.get_shape(), \n",
    "                                s_seq.get_shape(), tf.TensorShape([None, 100])])\n",
    "        \n",
    "        y_hidden_layer.append(output_seq)\n",
    "\n",
    "with tf.name_scope(\"dropout_hidden_layer_seq\"):\n",
    "    y_hidden_layer_drop = tf.nn.dropout(y_hidden_layer, 0.3)\n",
    "\n",
    "with tf.name_scope(\"softmax_layer_seq\"):\n",
    "    W = tf.Variable(tf.truncated_normal([100, 2], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([2]), name=\"b\")\n",
    "    \n",
    "    logits_entity = []\n",
    "    predictions_entity = []\n",
    "    lg = tf.zeros([1, 2])\n",
    "    \n",
    "    for batch in range(batch_size):\n",
    "        \n",
    "        def matmul_softmax(j, const, steps, y_hl, lg):\n",
    "            temp = tf.matmul(y_hl[j], W) + b\n",
    "            lg = tf.cond(tf.equal(j, const), lambda: temp, lambda: tf.concat([lg, temp], 0))\n",
    "            return j+1, const, steps, y_hl, lg\n",
    "\n",
    "        y_hl = tf.expand_dims(y_hidden_layer_drop[batch], 1)\n",
    "        \n",
    "        _, _, _, _, logit = tf.while_loop(cond, matmul_softmax, \n",
    "                                [0, 0, fp_length[batch], y_hl,lg],\n",
    "                                shape_invariants=[x.get_shape(), x.get_shape(), x.get_shape(), \n",
    "                                y_hl.get_shape(), tf.TensorShape([None, 2])])\n",
    "        \n",
    "        logits_entity.append(logit)\n",
    "        predictions_entity.append(tf.arg_max(logit, 1))\n",
    "        \n",
    "\n",
    "Y_en = [y_entity[i][:fp_length[i]] for i in range(batch_size)]\n",
    "\n",
    "with tf.name_scope(\"loss_seq\"):\n",
    "    loss_seq = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits_entity,\n",
    "                                                                        labels=Y_en))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"hidden_layer_dep\"):\n",
    "    W = tf.Variable(tf.truncated_normal([300, 100], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([100]), name=\"b\")\n",
    "    y_p = tf.tanh(tf.matmul(dp_series, W) + b)\n",
    "\n",
    "with tf.name_scope(\"softmax_layer_dep\"):\n",
    "    W = tf.Variable(tf.truncated_normal([100, relation_classes], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([relation_classes]), name=\"b\")\n",
    "    logits = tf.matmul(y_p, W) + b    \n",
    "    predictions_dep = tf.argmax(logits, 1)\n",
    "\n",
    "\n",
    "with tf.name_scope(\"loss_dep\"):\n",
    "    loss_dep = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=relation))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tv_all = tf.trainable_variables()\n",
    "\n",
    "tv_regu = []\n",
    "\n",
    "non_reg = [\"word_embedding/W:0\",\"pos_embedding/W:0\",'dep_embedding/W:0']\n",
    "\n",
    "for t in tv_all:\n",
    "    if t.name not in non_reg:\n",
    "        if t.name.find('b_')==-1:\n",
    "            if t.name.find('b:')==-1:\n",
    "                tv_regu.append(t)\n",
    "\n",
    "with tf.name_scope(\"total_loss\"):\n",
    "    l2_loss = lambda_l2 * tf.reduce_sum([ tf.nn.l2_loss(v) for v in tv_regu ])\n",
    "    total_loss = l2_loss + loss_seq + loss_dep\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False, name=\"global_step\")\n",
    "\n",
    "learning_rate = tf.train.exponential_decay(init_learning_rate, global_step, decay_steps, decay_rate, staircase=True)\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "optimizer_seq = tf.train.AdamOptimizer(learning_rate).minimize(loss_seq)\n",
    "\n",
    "grads_vars = optimizer.compute_gradients(total_loss)\n",
    "\n",
    "clipped_grads = [(tf.clip_by_norm(grad, gradient_clipping), var) for grad, var in grads_vars]\n",
    "train_op = optimizer.apply_gradients(clipped_grads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('data/vocab_glove', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "\n",
    "pos_tags_vocab = []\n",
    "for line in open('data/pos_tags.txt'):\n",
    "        pos_tags_vocab.append(line.strip())\n",
    "\n",
    "dep_vocab = []\n",
    "for line in open('data/dependency_types.txt'):\n",
    "    dep_vocab.append(line.strip())\n",
    "\n",
    "relation_vocab = []\n",
    "for line in open('data/relation_typesv3.txt'):\n",
    "    relation_vocab.append(line.strip())\n",
    "\n",
    "rel2id = dict((w, i) for i,w in enumerate(relation_vocab))\n",
    "id2rel = dict((i, w) for i,w in enumerate(relation_vocab))\n",
    "\n",
    "pos_tag2id = dict((w, i) for i,w in enumerate(pos_tags_vocab))\n",
    "id2pos_tag = dict((i, w) for i,w in enumerate(pos_tags_vocab))\n",
    "\n",
    "dep2id = dict((w, i) for i,w in enumerate(dep_vocab))\n",
    "id2dep = dict((i, w) for i,w in enumerate(dep_vocab))\n",
    "\n",
    "pos_tag2id['OTH'] = 9\n",
    "id2pos_tag[9] = 'OTH'\n",
    "\n",
    "dep2id['OTH'] = 20\n",
    "id2dep[20] = 'OTH'\n",
    "\n",
    "JJ_pos_tags = ['JJ', 'JJR', 'JJS']\n",
    "NN_pos_tags = ['NN', 'NNS', 'NNP', 'NNPS']\n",
    "RB_pos_tags = ['RB', 'RBR', 'RBS']\n",
    "PRP_pos_tags = ['PRP', 'PRP$']\n",
    "VB_pos_tags = ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "_pos_tags = ['CC', 'CD', 'DT', 'IN']\n",
    "\n",
    "def pos_tag(x):\n",
    "    if x in JJ_pos_tags:\n",
    "        return pos_tag2id['JJ']\n",
    "    if x in NN_pos_tags:\n",
    "        return pos_tag2id['NN']\n",
    "    if x in RB_pos_tags:\n",
    "        return pos_tag2id['RB']\n",
    "    if x in PRP_pos_tags:\n",
    "        return pos_tag2id['PRP']\n",
    "    if x in VB_pos_tags:\n",
    "        return pos_tag2id['VB']\n",
    "    if x in _pos_tags:\n",
    "        return pos_tag2id[x]\n",
    "    else:\n",
    "        return 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_input(words_seq, deps_seq, pos_tags_seq, word_path1, word_path2, dep_path1, dep_path2, pos_tags_path1, pos_tags_path2, pos_path1,pos_path2, childs_path1, childs_path2, relations):\n",
    "\n",
    "    length = len(words_seq)\n",
    "    \n",
    "    pos_path1 = [[i-1 for i in w] for w in pos_path1]\n",
    "    pos_path2 = [[i-1 for i in w] for w in pos_path2]\n",
    "\n",
    "    for i in range(length):\n",
    "            words_seq[i] = [w for w in words_seq[i] if w !=None ]\n",
    "            deps_seq[i] = [w for w in deps_seq[i] if w !=None ]\n",
    "            pos_tags_seq[i] = [w for w in pos_tags_seq[i] if w !=None ]\n",
    "\n",
    "    entity = np.zeros([length, max_len_seq])\n",
    "    for i in range(length):\n",
    "        entity[i][pos_path1[i][0]] = 1\n",
    "        if(pos_path2[i]==[]):\n",
    "            entity[i][pos_path1[i][-1]] = 1\n",
    "        else:\n",
    "            entity[i][pos_path2[i][0]] = 1\n",
    "\n",
    "    len_path1 = []\n",
    "    len_path2 = []\n",
    "\n",
    "    num_child_path1 = np.ones([length, max_len_path],dtype=int) * -1\n",
    "    num_child_path2 = np.ones([length, max_len_path],dtype=int) * -1\n",
    "\n",
    "    for w in word_path1:\n",
    "        len_path1.append(len(w))\n",
    "\n",
    "    for w in word_path2:\n",
    "        len_path2.append(len(w))\n",
    "\n",
    "    for i, w in enumerate(childs_path1):\n",
    "        if(w!=[]):\n",
    "            for j,c in enumerate(w):\n",
    "                num_child_path1[i][j] = len(c)\n",
    "        else:\n",
    "            num_child_path1[i][0] = 0\n",
    "\n",
    "    for i, w in enumerate(childs_path2):\n",
    "        if(w!=[]):\n",
    "            for j,c in enumerate(w):\n",
    "                num_child_path2[i][j] = len(c)\n",
    "        else:\n",
    "            num_child_path2[i][0] = 0\n",
    "\n",
    "    for i in range(length):\n",
    "        if(childs_path2[i]!=[]):\n",
    "            for j, c in enumerate(childs_path2[i]):\n",
    "                if(c == []):\n",
    "                    childs_path2[i][j]  = [-1]\n",
    "        else:\n",
    "            childs_path2[i] = [[-1]]\n",
    "            \n",
    "        if(childs_path1[i]!=[]):\n",
    "            for j, c in enumerate(childs_path1[i]):\n",
    "                if(c == []):\n",
    "                    childs_path1[i][j]  = [-1]\n",
    "        else:\n",
    "            childs_path1[i] = [[-1]]\n",
    "\n",
    "    for i in range(length):\n",
    "        if(word_path2[i]==[]):\n",
    "            word_path2[i].append(unknown_token)\n",
    "        if(dep_path2[i]==[]):\n",
    "            dep_path2[i].append('OTH')\n",
    "        if(pos_tags_path2==[]):\n",
    "            pos_tags_path2[i].append('OTH')\n",
    "\n",
    "    for i in range(length):\n",
    "        for j, word in enumerate(words_seq[i]):\n",
    "            word = word.lower()\n",
    "            words_seq[i][j] = word if word in word2id else unknown_token \n",
    "\n",
    "        for l, d in enumerate(deps_seq[i]):\n",
    "            deps_seq[i][l] = d if d in dep2id else 'OTH'\n",
    "\n",
    "        for j, word in enumerate(word_path1[i]):\n",
    "            word = word.lower()\n",
    "            word_path1[i][j] = word if word in word2id else unknown_token \n",
    "\n",
    "        for l, d in enumerate(dep_path1[i]):\n",
    "            dep_path1[i][l] = d if d in dep2id else 'OTH'\n",
    "\n",
    "        for j, word in enumerate(word_path2[i]):\n",
    "            word = word.lower()\n",
    "            word_path2[i][j] = word if word in word2id else unknown_token \n",
    "\n",
    "        for l, d in enumerate(dep_path2[i]):\n",
    "            dep_path2[i][l] = d if d in dep2id else 'OTH'\n",
    "    \n",
    "    words_seq_id = np.ones([length, max_len_seq],dtype=int) * -1\n",
    "    deps_seq_id = np.ones([length, max_len_seq],dtype=int) * -1\n",
    "    pos_tags_seq_id = np.ones([length, max_len_seq],dtype=int) * -1\n",
    "    \n",
    "    word_path1_id = np.ones([length, max_len_path],dtype=int) * -1\n",
    "    word_path2_id = np.ones([length, max_len_path],dtype=int) * -1\n",
    "    \n",
    "    dep_path1_id = np.ones([length, max_len_path],dtype=int) * -1\n",
    "    dep_path2_id = np.ones([length, max_len_path],dtype=int) * -1\n",
    "    \n",
    "    pos_tags_path1_id = np.ones([length, max_len_path],dtype=int) * -1\n",
    "    pos_tags_path2_id = np.ones([length, max_len_path],dtype=int) * -1\n",
    "    \n",
    "    pos_path1_ = np.ones([length, max_len_path],dtype=int) * -1\n",
    "    pos_path2_ = np.ones([length, max_len_path],dtype=int) * -1\n",
    "    \n",
    "    childs_path1_ = np.ones([length, max_len_path, max_num_childs],dtype=int) * -1\n",
    "    childs_path2_ = np.ones([length, max_len_path, max_num_childs],dtype=int) * -1\n",
    "    \n",
    "    seq_len = []\n",
    "\n",
    "    for i in range(length):\n",
    "        \n",
    "        temp = []\n",
    "        seq_len.append(len(words_seq[i]))\n",
    "\n",
    "        for j, w in enumerate(pos_path1[i]):\n",
    "            pos_path1_[i][j] = w\n",
    "        \n",
    "        for j, w in enumerate(pos_path2[i]):\n",
    "            pos_path2_[i][j] = w\n",
    "        \n",
    "        for j,child in enumerate(childs_path1[i]):\n",
    "            for k,c in enumerate(child):\n",
    "                childs_path1_[i][j][k] = c\n",
    "                \n",
    "        for j,child in enumerate(childs_path2[i]):\n",
    "            for k,c in enumerate(child):\n",
    "                childs_path2_[i][j][k] = c    \n",
    "    \n",
    "        for j, w in enumerate(words_seq[i]):\n",
    "            words_seq_id[i][j] = word2id[w]\n",
    "\n",
    "        \n",
    "        for j, w in enumerate(pos_tags_seq[i]):\n",
    "            pos_tags_seq_id[i][j] = pos_tag(w)\n",
    "        \n",
    "\n",
    "        \n",
    "        for j, w in enumerate(deps_seq[i]):\n",
    "            deps_seq_id[i][j] = dep2id[w]\n",
    "         \n",
    "\n",
    "        \n",
    "        for j, w in enumerate(word_path1[i]):\n",
    "            word_path1_id[i][j]   = word2id[w]\n",
    "         \n",
    "\n",
    "        \n",
    "        for j, w in enumerate(pos_tags_path1[i]):\n",
    "            pos_tags_path1_id[i][j] = pos_tag(w)\n",
    "        \n",
    "        \n",
    "        for j, w in enumerate(dep_path1[i]):\n",
    "            dep_path1_id[i][j] = dep2id[w]\n",
    "         \n",
    "\n",
    "        \n",
    "        for j, w in enumerate(word_path2[i]):\n",
    "            word_path2_id[i][j] = word2id[w]\n",
    "       \n",
    "\n",
    "        \n",
    "        for j, w in enumerate(pos_tags_path2[i]):\n",
    "            pos_tags_path2_id[i][j] = pos_tag(w)\n",
    "         \n",
    "\n",
    "        \n",
    "        for j, w in enumerate(dep_path2[i]):\n",
    "            dep_path2_id[i][j]  = dep2id[w]\n",
    "         \n",
    "\n",
    "    rel_ids = np.array([rel2id[rel] for rel in relations])\n",
    "    \n",
    "    return seq_len, words_seq_id, pos_tags_seq_id, deps_seq_id, len_path1, len_path2, pos_path1_, pos_path2_, dep_path1_id, dep_path2_id, childs_path1_, childs_path2_, num_child_path1, num_child_path2, rel_ids, entity "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('data/word_embedding', 'rb')\n",
    "# word_embedding = pickle.load(f)\n",
    "# f.close()\n",
    "# sess.run(embedding_init, feed_dict={embedding_placeholder:word_embedding})\n",
    "# word_embedding_saver.save(sess, word_embd_dir + '/word_embd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# model = tf.train.latest_checkpoint(model_dir)\n",
    "# saver.restore(sess, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "latest_embd = tf.train.latest_checkpoint(word_embd_dir)\n",
    "word_embedding_saver.restore(sess, latest_embd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('data/train_pathsv3', 'rb')\n",
    "words_seq, deps_seq, pos_tags_seq, word_path1, word_path2, dep_path1, dep_path2, pos_tags_path1, pos_tags_path2, pos_path1, pos_path2, childs_path1, childs_path2 = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "relations = []\n",
    "for line in open('data/train_relationsv3.txt'):\n",
    "    relations.append(line.strip().split()[1])\n",
    "\n",
    "length = len(words_seq)\n",
    "num_batches = int(length/batch_size)\n",
    "\n",
    "seq_len, words_seq_id, pos_tags_seq_id, deps_seq_id, len_path1, len_path2, pos_path1, pos_path2, dep_path1_id, dep_path2_id, childs_path1, childs_path2, num_child_path1, num_child_path2, rel_ids, entity = prepare_input(words_seq, deps_seq, pos_tags_seq, word_path1, word_path2, dep_path1, dep_path2, pos_tags_path1, pos_tags_path2, pos_path1,pos_path2, childs_path1, childs_path2, relations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for i, word in enumerate(words_seq):\n",
    "#     if i==0:\n",
    "#         freq = nltk.FreqDist(word)\n",
    "#     else:\n",
    "#         freq += nltk.FreqDist(word)\n",
    "# most_common_words = freq.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 1\n",
    "\n",
    "for i in range(num_epochs):\n",
    "    \n",
    "    loss_per_epoch = 0\n",
    "    \n",
    "    for j in range(num_batches):\n",
    "        \n",
    "        s = j * batch_size\n",
    "        end = (j+1) * batch_size\n",
    "        \n",
    "        feed_dict = {\n",
    "            fp_length: seq_len[s:end],\n",
    "            fp: [[words_seq_id[k], pos_tags_seq_id[k]] for k in range(s, end)],\n",
    "            sp_length: [[len_path1[k], len_path2[k]] for k in range(s, end)],\n",
    "            sp: [[dep_path1_id[k],dep_path2_id[k]] for k in range(s, end)],\n",
    "            sp_pos : [[pos_path1[k], pos_path2[k]] for k in range(s, end)],\n",
    "            sp_childs: [[childs_path1[k], childs_path2[k]] for k in range(s, end)],\n",
    "            sp_num_childs: [[num_child_path1[k], num_child_path2[k]] for k in range(s, end)],\n",
    "            relation: rel_ids[s:end],\n",
    "            y_entity: entity[s:end]}\n",
    "        \n",
    "        _, _loss, step = sess.run([optimizer_seq, loss_seq, global_step], feed_dict)\n",
    "        \n",
    "        loss_per_epoch +=_loss\n",
    "        \n",
    "#         if (j+1)%num_batches==0:\n",
    "        print(\"Epoch:\", i+1,\"Step:\", step, \"loss:\",loss_per_epoch/num_batches)\n",
    "            \n",
    "    saver.save(sess, model_dir + '/model')\n",
    "    print(\"Saved Model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# training accuracy\n",
    "all_predictions = []\n",
    "for j in range(num_batches):\n",
    "    s = j * batch_size\n",
    "    end = (j+1) * batch_size\n",
    "\n",
    "    feed_dict = {\n",
    "        fp_length: seq_len[s:end],\n",
    "        fp: [[words_seq_id[s], pos_tags_seq_id[end]] for k in range(s, end)],\n",
    "        sp_length: [[len_path1[k], len_path2[k]] for k in range(s, end)],\n",
    "        sp: [[dep_path1_id[k],dep_path2_id[k]] for k in range(s, end)],\n",
    "        sp_pos : [[pos_path1[k], pos_path2[k]] for k in range(s, end)],\n",
    "        sp_childs: [[childs_path1[k], childs_path2[k]] for k in range(s, end)],\n",
    "        sp_num_childs: [[num_child_path1[k], num_child_path2[k]] for k in range(s, end)],\n",
    "        relation: rel_ids[s:end],\n",
    "        y_entity: entity[s:end]}\n",
    "        \n",
    "    batch_predictions = sess.run(predictions, feed_dict)\n",
    "    all_predictions.append(batch_predictions)\n",
    "\n",
    "y_pred = []\n",
    "for i in range(num_batches):\n",
    "    for pred in all_predictions[i]:\n",
    "        y_pred.append(pred)\n",
    "\n",
    "count = 0\n",
    "for i in range(batch_size*num_batches):\n",
    "    count += y_pred[i]==rel_ids[i]\n",
    "accuracy = count/(batch_size*num_batches) * 100\n",
    "\n",
    "print(\"training accuracy\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('data/test_pathsv3', 'rb')\n",
    "words_seq, deps_seq, pos_tags_seq, word_path1, word_path2, dep_path1, dep_path2, pos_tags_path1, pos_tags_path2, pos_path1, pos_path2, childs_path1, childs_path2 = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "relations = []\n",
    "for line in open('data/test_relationsv3.txt'):\n",
    "    relations.append(line.strip().split()[1])\n",
    "\n",
    "length = len(words_seq)\n",
    "num_batches = int(length/batch_size)\n",
    "\n",
    "seq_len, words_seq_id, pos_tags_seq_id, deps_seq_id, len_path1, len_path2, dep_path1_id, dep_path2_id, childs_path1, childs_path2, num_child_path1, num_child_path2, rel_ids, entity = prepare_input(words_seq, deps_seq, pos_tags_seq, word_path1, word_path2, dep_path1, dep_path2, pos_tags_path1, pos_tags_path2, pos_path1,pos_path2, childs_path1, childs_path2, relations)\n",
    "\n",
    "# test predictions\n",
    "\n",
    "all_predictions = []\n",
    "for j in range(num_batches):\n",
    "    s = j * batch_size\n",
    "    end = (j+1) * batch_size\n",
    "\n",
    "    feed_dict = {\n",
    "        fp_length: seq_len[s:end],\n",
    "        fp: [[words_seq_id[s], pos_tags_seq_id[end]] for k in range(s, end)],\n",
    "        sp_length: [[len_path1[k], len_path2[k]] for k in range(s, end)],\n",
    "        sp: [[dep_path1_id[k],dep_path2_id[k]] for k in range(s, end)],\n",
    "        sp_pos : [[pos_path1[k], pos_path2[k]] for k in range(s, end)],\n",
    "        sp_childs: [[childs_path1[k], childs_path2[k]] for k in range(s, end)],\n",
    "        sp_num_childs: [[num_child_path1[k], num_child_path2[k]] for k in range(s, end)],\n",
    "        relation: rel_ids[s:end],\n",
    "        y_entity: entity[s:end]}\n",
    "        \n",
    "    batch_predictions = sess.run(predictions, feed_dict)\n",
    "    all_predictions.append(batch_predictions)\n",
    "\n",
    "y_pred = []\n",
    "\n",
    "for i in range(num_batches):\n",
    "    for pred in all_predictions[i]:\n",
    "        y_pred.append(pred)\n",
    "\n",
    "count = 0\n",
    "\n",
    "for i in range(batch_size*num_batches):\n",
    "    count += y_pred[i]==rel_ids[i]\n",
    "accuracy = count/(batch_size*num_batches) * 100\n",
    "\n",
    "print(\"test accuracy\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
